package com.gome.ss.flowcrawl.extractor;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;

public class RemoveDuplicate {
	
	public HTMLCount hCount = new HTMLCount();
	public HashDict  hDict = new HashDict();
	public final float RATIO = 0.7f;
	public final int LISTSIZE = 40;
	private HashMap<String, BitMap> hb = new HashMap<String, BitMap>();
	
	public RemoveDuplicate() {
		
	}
	
	public void putDoc(String index, String doc)throws Exception 
	{
		List<String>  ls = hCount.count2(doc);
		if(ls.size()<LISTSIZE)
			return;
		
		Set<Integer> idset = new HashSet<Integer>();
		int maxID = 0;
		for(String s : ls)
		{
			int id = hDict.add(s);
			if (id > maxID)
				maxID = id;
			idset.add(id);
		}
		
		BitMap bm = new BitMap(maxID);
		
		for (Integer id : idset) {
			bm.SetBit(id);
		}
		
		hb.put(index, bm);
	}
	
	public Set<String> getRemoveDupSet() {
		Set<String> h = new HashSet<String>();
		
		for(Entry<String, BitMap> entry : hb.entrySet())
		{
			String id = entry.getKey();
			
			if(h.contains(id))
				continue;
			
			BitMap ba = entry.getValue();
			byte[] a = ba.Get();
			for(Entry<String, BitMap> sentry : hb.entrySet())
			{
				String sid = sentry.getKey();
				if (id.equals(sid))
					continue;
				
				if(h.contains(sid))
					continue;
				
				BitMap bb = sentry.getValue();
				byte[] b = bb.Get();
				
				int num = max(ba.GetBitCount(), bb.GetBitCount());
				int sc = BitMap.calc(a, b);
				if(sc==0)
					continue;
				double f = sc*1.0/num;	
				if(f>RATIO)
				{
					h.add(sid);
				}
			}
		}
		
		return h;
	}
	
	public int max(int a, int b)
	{
		return a>b?a:b;
	}
}